package com.chance.cc.crawler.development.scripts.bitauto.carType;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.DigestUtils;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.*;

/**
 * @ClassName song
 * @Description TODO
 * @Author ding
 * @Date 2021/9/6 13:25
 * @Version 1.0
 **/
public class BitAutoCarTypeCrawlerScript extends CrawlerCommonScript {
    private static Logger log = LoggerFactory.getLogger(BitAutoCarTypeCrawlerScript.class);
    private static final String domain = "bitauto";
    private static final String site = "carType";
    private static final String RECORD_AGAIN_REQUEST = "record_again_request";

    private static final String HTTPS = "https:";
    private static final String BrandUrl = "https://car.yiche.com";
    private Downloader httpDownload;
    private static final Object httpDownloadObj = new Object();


    private static final String seriesUrl = "https://car.yiche.com/\\S*/";
    private static final String seriesConfig = "https://car.yiche.com/web_api/car_model_api/api/v1/car/car_list_condition?cid=508&param=%s";
    private static final String seriesConfigUrl = "https://car.yiche.com/web_api/car_model_api/api/v1/car\\S*";
    private static final Map<String,String> headMap = new HashMap<>();
    @Override
    public void initUrlRegulars() {
        addUrlRegular(seriesUrl);
        addUrlRegular("https://car.yiche.com/");
        addUrlRegular(seriesConfigUrl);
    }
    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> listRecord = new ArrayList<>();
        String seriesUrl = requestRecord.getHttpRequest().getUrl();
        if (supportSourceRecords != null){
            for (CrawlerRequestRecord supportSourceRecord :supportSourceRecords){
                String url = supportSourceRecord.getHttpRequest().getUrl();
                if (url.contains("keys")){
                    HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
                    Json json = internalDownloadPage.getJson();
                    String msg = json.jsonPath($_type+".msg").get();
                    if (!"success".equals(msg)){
                        log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
                        return listRecord;
                    }
                    List<String> all = json.jsonPath($_type + ".content").all();
                    for (String data : all) {
                        JSONObject jsonObject = JSONObject.parseObject(data);
                        String keyword = jsonObject.getString("keyword");
                        String startUrl = BrandUrl+"/"+keyword+"/";
                        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                .itemPageRequest(requestRecord)
                                .httpUrl(startUrl)
                                .releaseTime(System.currentTimeMillis())
                                .copyResultTags()
                                .copyBizTags()
                                .build();
                        listRecord.add(record);
                    }
                }
            }
        }
        if (seriesUrl.matches(seriesConfigUrl)){
            String seriesId = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("seriesId");
            String signAndTime = this.getSign(seriesId);
            String[] split1 = signAndTime.split("/");
            String sign = split1[0];
            String time = split1[1];
            headMap.put("gudslf",getJiami());
            headMap.put("gudpar",getJiami());
            headMap.put("reqid",getJiami());
            headMap.put("x-sign",sign);
            headMap.put("Referer",requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("seriesUrl"));
            headMap.put("x-timestamp", time);
            requestRecord.getHttpRequest().setHeaders(headMap);
            requestRecord.getHttpRequest().setCookies(getCookie());
        }
        return listRecord;
    }
    @Override
    public void beforeDownload(CrawlerRecordContext context) {
        if (httpDownload == null){
            synchronized (httpDownloadObj) {
                if (httpDownload == null){
                    httpDownload = context.getPageDownloader();
                }
            }
        }
        super.beforeDownload(context);
    }


    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parseLinks = new ArrayList<>();
        if (page.getStatusCode() != 200 ||  !page.isDownloadSuccess()){
            log.error("page == [{}] || statusCode != 200 and error page = "+ page.getStatusCode());
            if (page.getStatusCode() != 404){
                recordAgainRequest(crawlerRecord,parseLinks);
                crawlerRecord.setNeedWashPage(false);
                return  parseLinks;
            }else{
                crawlerRecord.setNeedWashPage(false);
                return  parseLinks;
            }
        }
        String url = crawlerRecord.getHttpRequest().getUrl();
        if (url.matches(seriesUrl)){
            String story = page.getHtml().xpath("//*[@id=\"shortBrandStory\"]//text()").get();
            if (story!=null){
                //this.parseSeriesUrl(crawlerRecord,page,parseLinks);
            }else{
                crawlerRecord.setNeedWashPage(true);
                this.parseSeries(crawlerRecord,page,parseLinks);
            }

        }
        if (url.matches(seriesConfigUrl)){
            crawlerRecord.setNeedWashPage(true);
        }
        return parseLinks;
    }

    private void parseSeriesUrl(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"data_table_MasterSerialList_0\"]/div").nodes();
        for (Selectable node :nodes){
            String href = node.xpath("./div/a/@href").get();
            String url = BrandUrl + href;
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            crawlerRecord.setNeedWashPage(false);
            parseLinks.add(record);
        }
    }

    /*
    * 得到车型的链接
    * */
    private void parseSeries(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        this.message(crawlerRecord,page);

        List<Selectable> nodes = page.getHtml().xpath("/html/body/div[@class=\"content chexin clearfix\"]/div[1]/div[2]/div[1]/ul/li").nodes();
        if (nodes.size()==1){
            return;
        }
        String s = page.getHtml().xpath("/html/body/div[@class=\"content chexin clearfix\"]/div[1]/div[1]/div[1]/div[1]/a[1]/@href").get();
        if (s == null){
            log.error("无法获取车系id");
            return;
        }
        String[] split = s.split("/");
        String seriesId = split[split.length-1];
        String param = "{\"serialId\":\""+seriesId+"\"}";
        param = URLEncoder.encode(param);
        String url = String.format(seriesConfig,param);
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .releaseTime(System.currentTimeMillis())
                .httpUrl(url)
                .copyBizTags()
                .copyResultTags()
                .build();
        record.getHttpRequest().setMethod("get");
        record.tagsCreator().bizTags().addCustomKV("seriesId",seriesId);
        record.tagsCreator().bizTags().addCustomKV("seriesUrl",crawlerRecord.getHttpRequest().getUrl());
        parseLinks.add(record);
    }


    private String getSign(String seriesId) {
        String time = String.valueOf(System.currentTimeMillis());
        String s  = "cid=508&param={\"serialId\":\""+seriesId+"\"}19DDD1FBDFF065D3A4DA777D2D7A81EC"+time;
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpConfig httpConfig = HttpConfig.me(domain);
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setResponseCharset("UTF-8");
        httpRequest.setMethod("post");
        httpRequest.setUrl("http://192.168.1.210:8899/encrypt/sign");
        Map<String,Object> bodyMap = new HashMap<>();
        bodyMap.put("params",s);
        HttpRequestBody form = HttpRequestBody.form(bodyMap, "UTF-8");
        httpRequest.setRequestBody(form);
        HttpPage download = downloader.download(httpRequest, httpConfig);
        String rawText = download.getRawText();
        return rawText+"/"+time;
    }

    private String getJiami() {
        String s = this.getcookieString();
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpConfig httpConfig = HttpConfig.me(domain);
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setResponseCharset("UTF-8");
        httpRequest.setMethod("post");
        httpRequest.setUrl("http://192.168.1.210:8899/encrypt/bitauto");
        Map<String,Object> bodyMap = new HashMap<>();
        bodyMap.put("params",s);
        HttpRequestBody form = HttpRequestBody.form(bodyMap, "UTF-8");
        httpRequest.setRequestBody(form);
        HttpPage download = downloader.download(httpRequest, httpConfig);
        String rawText = download.getRawText();
        return rawText;
    }

    private Map<String,String> getCookie(){
        Map<String,String> cookieMap = new HashMap<>();
        cookieMap.put("cookieNo","1");
        cookieMap.put("XCWEBLOG_testcookie","yes");
        cookieMap.put("CIGDCID","b4053429655444d1b51529842c712325");
        cookieMap.put("G_CIGDCID","b4053429655444d1b51529842c712325");
        cookieMap.put("CIGUID","2b602241-5141-41ea-89c0-6a1817ccbbe6");
        cookieMap.put("auto_id","784680a2956e4acd8f6ccdeaad025bcc");
        cookieMap.put("UserGuid","2b602241-5141-41ea-89c0-6a1817ccbbe6");
        cookieMap.put("selectcity","310100");
        cookieMap.put("selectcityid","2401");
        cookieMap.put("selectcityName","%E4%B8%8A%E6%B5%B7");
        cookieMap.put("locatecity","310100");
        cookieMap.put("bitauto_ipregion","180.164.52.239%3A%E4%B8%8A%E6%B5%B7%E5%B8%82%3B2401%2C%E4%B8%8A%E6%B5%B7%2Cshanghai");
        cookieMap.put("Hm_lvt_610fee5a506c80c9e1a46aa9a2de2e44","1630905245,1631169684,1631497983,"+String.valueOf(System.currentTimeMillis()).substring(0,10));
        cookieMap.put("csids","2408_7480_3152_1796_1798_1594_6270_5779_7348_5131");
        cookieMap.put("report-cookie-id","575197559_"+System.currentTimeMillis());
        cookieMap.put("Hm_lpvt_610fee5a506c80c9e1a46aa9a2de2e44",String.valueOf(System.currentTimeMillis()).substring(0,10));
        return cookieMap;
    }
    private String getcookieString() {
        Map<String,String> map = getCookie();
        StringBuffer content = new StringBuffer();
        for(String key : map.keySet()){
            content.append(key).append("=").append(map.get(key)).append("; ");
        }
        return String.valueOf(content);
    }

    /**
     * @param url 请求Url
     * @return header的sign值
     */
    private String generatorSignFromUrl(String url, String timestamp){
        Map<String, Object> urlParams = getUrlParams(url);
        try {
            if (null != urlParams){
                String u = "19DDD1FBDFF065D3A4DA777D2D7A81EC";
                String cid = (String) urlParams.get("cid");
                String param = URLDecoder.decode((String) urlParams.get("param"),"utf-8");
                String s = "cid=" + cid + "&param=" + param + u + timestamp;
                return DigestUtils.md5DigestAsHex(s.getBytes());
            }
        } catch (UnsupportedEncodingException e) {
            log.error(e.getMessage(),"url decode error");
        }
        return UUID.randomUUID().toString().replaceAll("-","");
    }
    /**
     * 将url参数转换成map
     * @param url http://*.*.com?aa=11&bb=22&cc=33
     * @return map
     */
    private Map<String, Object> getUrlParams(String url) {
        Map<String, Object> map = new HashMap<String, Object>(0);
        String param = null;
        if (url.contains("?")){
            param = url.split("\\?")[1];
        }
        if (StringUtils.isBlank(param)) {
            return null;
        }
        String[] params = param.split("&");
        for (String s : params) {
            String[] p = s.split("=");
            if (p.length == 2) {
                map.put(p[0], p[1]);
            }
        }
        return map;
    }
    static {
        headMap.put("Connection","keep-alive");
        headMap.put("Pragma","no-cache");
        headMap.put("Cache-Control","no-cache");
        headMap.put("sec-ch-ua","\"Chromium\";v=\"92\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"92\"");
        headMap.put("dvid","2b602241-5141-41ea-89c0-6a1817ccbbe6");
        headMap.put("x-platform","pc");
        headMap.put("osvl","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36");
        headMap.put("x-requested-with","XMLHttpRequest");
        headMap.put("uidl","");
        headMap.put("x-city-id","3101");
        headMap.put("osl","3");
        headMap.put("sec-ch-ua-mobile","?0");
        headMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36");
        headMap.put("accept","application/json, text/javascript, */*; q=0.01");
        headMap.put("x-ip-address","180.164.52.239");
        headMap.put("x-user-guid","2b602241-5141-41ea-89c0-6a1817ccbbe6");
        headMap.put("ver","v10.58.3");
        headMap.put("Sec-Fetch-Site","same-origin");
        headMap.put("Sec-Fetch-Mode","cors");
        headMap.put("Sec-Fetch-Dest","empty");
        headMap.put("Accept-Language","zh-CN,zh;q=0.9");
        headMap.put("sec-ch-ua-platform ","Windows");
    }
    private void message(CrawlerRequestRecord crawlerRecord, HttpPage page){
        String brandName = page.getHtml().xpath("/html/body/div[4]/div[1]/div/div[4]/a/text()").get();
        String brandId = page.getHtml().xpath("/html/body/div[4]/div[1]/div/div[4]/a/@href").get();
        String brandUrl = BrandUrl + brandId;
        brandId = brandId.replaceAll("/","");
        String carTypeName = page.getHtml().xpath("/html/body/div[4]/div[1]/div/div[5]/span/text()").get();
        String carTypeUrl = crawlerRecord.getHttpRequest().getUrl();
        String[] split1 = carTypeUrl.split("/");
        String carTypeId = split1[split1.length-1];
        String koubeiUrl = "https://dianping.yiche.com/"+carTypeId+"/koubei/";
        String wendaUrl = "https://wenda.yiche.com/"+carTypeId+"/ask/";
        Map<String,String> map = new HashMap<>();
        map.put("brandName",brandName);
        map.put("brandId",brandId);
        map.put("brandUrl",brandUrl);
        map.put("carTypeName",carTypeName);
        map.put("carTypeUrl",carTypeUrl);
        map.put("carTypeId",carTypeId);
        map.put("koubeiUrl",koubeiUrl);
        map.put("wendaUrl",wendaUrl);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("message",map);
    }
    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> listData = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if (url.matches(seriesUrl)){
            String story = page.getHtml().xpath("//*[@id=\"shortBrandStory\"]//text()").get();//判断是否是品牌或者配置
            if (story != null){
                this.washBrandMessage(crawlerRecord,page,listData);//清洗品牌信息
            }else{
                this.washSeriesArticle(crawlerRecord,page,listData);//清洗不存在停售款的
            }

        }
        if (url.matches(seriesConfigUrl)){
            this.washArticle(crawlerRecord,page,listData);
        }
        return listData;
    }

    private void washBrandMessage(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> listData) {
        String brandName = page.getHtml().xpath("/html/body/div[3]/div[1]/h1/span/text()").get();
        if (brandName.contains("&amp;middot;")){
           brandName =  brandName.replaceAll("&amp;middot;","·");
        }
        String text = page.getHtml().xpath("/html/body/script[19]/text()").get();
        String[] split = text.split("carList =");
        String json = split[1];
        String substring = json.substring(0, json.length() - 1);
        page.setRawText(substring);
        List<String> onAndWaitList = page.getJson().jsonPath($_type + ".onAndWaitList").all();//在售款
        for (Object object : onAndWaitList) {
            JSONObject brandObject = JSONObject.parseObject(String.valueOf(object));
            String brandChildName = brandObject.getString("name");//子品牌名
            JSONArray serialList = brandObject.getJSONArray("serialList");
            for (Object obj : serialList) {
                JSONObject jsonObject = JSONObject.parseObject(String.valueOf(obj));
                String car_id = jsonObject.getString("id");
                String car_name = jsonObject.getString("name");
                String allSpell = jsonObject.getString("allSpell");
                String is_sale = "1";
                JSONObject carMarketJson = jsonObject.getJSONObject("carMarket");
                String car_market = null;
                if (carMarketJson == null){
                    car_market = "null";
                }else{
                    car_market = carMarketJson.getString("value");
                }
                int i = new Random().nextInt(1000000000);
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article, i))
                        .url(crawlerRecord.getHttpRequest().getUrl())
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .addContentKV(AICCommonField.Field_Urls, "https://car.yiche.com/" + allSpell + "/")
                        .addContentKV(AutoVMCommonField.Field_Brand_name, brandName)
                        .addContentKV(AutoVMCommonField.Tag_Field_Brand_Child, brandChildName)
                        .addContentKV(AutoVMCommonField.Field_Car_Id, car_id)
                        .addContentKV(AutoVMCommonField.Field_Series_name, car_name)
                        .addContentKV("allSpell", allSpell)
                        .addContentKV(AutoVMCommonField.Tag_Is_sale, is_sale)
                        .addContentKV(AutoVMCommonField.Field_Car_Market, car_market)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                listData.add(crawlerData);
            }
        }
        List<String> unAndStopList = page.getJson().jsonPath($_type + ".unAndStopList").all();//未售和停售款
        for (Object object : unAndStopList) {
            JSONObject brandObject = JSONObject.parseObject(String.valueOf(object));
            String brandChildName = brandObject.getString("name");//子品牌名
            JSONArray serialList = brandObject.getJSONArray("serialList");
            for (Object obj : serialList) {
                JSONObject jsonObject = JSONObject.parseObject(String.valueOf(obj));
                String car_id = jsonObject.getString("id");
                String car_name = jsonObject.getString("name");
                String allSpell = jsonObject.getString("allSpell");
                String saleStatus = jsonObject.getString("saleStatus");
                if (saleStatus.equals("4")){
                    saleStatus = "0";
                }else if(saleStatus.equals("2")){
                    saleStatus = "-1";
                }else{
                    saleStatus = "0";
                }
                int i = new Random().nextInt(1000000000);
                JSONObject carMarketJson = jsonObject.getJSONObject("carMarket");
                String car_market = null;
                if (carMarketJson == null){
                    car_market = "null";
                }else{
                    car_market = carMarketJson.getString("value");
                }
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article, i))
                        .url(crawlerRecord.getHttpRequest().getUrl())
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .addContentKV(AICCommonField.Field_Urls, "https://car.yiche.com/" + allSpell + "/")
                        .addContentKV(AutoVMCommonField.Field_Brand_name, brandName)
                        .addContentKV(AutoVMCommonField.Tag_Field_Brand_Child, brandChildName)
                        .addContentKV(AutoVMCommonField.Field_Car_Id, car_id)
                        .addContentKV(AutoVMCommonField.Field_Series_name, car_name)
                        .addContentKV(AutoVMCommonField.Field_Brand_url, allSpell)
                        .addContentKV(AutoVMCommonField.Tag_Is_sale, saleStatus)
                        .addContentKV(AutoVMCommonField.Field_Car_Market, car_market)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                listData.add(crawlerData);
            }
        }
    }

    private void washSeriesArticle(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> listData) {
        List<Selectable> nodes = page.getHtml().xpath("/html/body/div[@class=\"content chexin clearfix\"]/div[1]/div[2]/div[1]/ul/li").nodes();
        if (nodes.size()>1){
            return;
        }
        KVTag map = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("message");
        crawlerRecord.tagsCreator().bizTags().addCustomKV("message",null);
        Map<String,String> val = (Map<String, String>) map.getVal();
        String brandName = val.get("brandName");
        String brandId = val.get("brandId");
        String brandUrl = val.get("brandUrl");
        String carTypeName = val.get("carTypeName");
        String carTypeUrl = val.get("carTypeUrl");
        String carTypeId = val.get("carTypeId");
        String koubeiUrl = val.get("koubeiUrl");
        String wendaUrl = val.get("wendaUrl");
        List<Selectable> tableNode = page.getHtml().xpath("/html/body/div[@class=\"content chexin clearfix\"]/div[1]/div[2]/div[2]/div/table").nodes();
        for (Selectable s : tableNode){
            List<Selectable> all = s.xpath("./tbody/tr").nodes();
            for (Selectable able : all){
                List<String> count = able.xpath("./td[1]/a/text()").all();
                StringBuffer content = new StringBuffer();
                for (String str : count){
                    content.append(str);
                }
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,new Random().nextInt(100000000)))
                        .releaseTime(System.currentTimeMillis())
                        .url(crawlerRecord.getHttpRequest().getUrl())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .addContentKV(AutoVMCommonField.Field_Brand_url,brandUrl)
                        .addContentKV(AutoVMCommonField.Field_Brand_name,brandName)
                        .addContentKV(AutoVMCommonField.Field_Brand_id,brandId)
                        .addContentKV(AutoVMCommonField.Field_Series_name,carTypeName)
                        .addContentKV(AutoVMCommonField.Field_Series_url,carTypeUrl)
                        .addContentKV(AutoVMCommonField.Field_Series_id,carTypeId)
                        .addContentKV(AutoVMCommonField.Tag_Is_sale,"1")
                        .addContentKV(AutoVMCommonField.Field_Praise_url,koubeiUrl)
                        .addContentKV(AutoVMCommonField.Field_Forum_url,wendaUrl)
                        .addContentKV(AutoVMCommonField.Field_Forum_name,carTypeName+"论坛")
                        .addContentKV(AICCommonField.Field_Content, String.valueOf(content))
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                listData.add(crawlerData);
            }
        }
    }

    private void washArticle(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> listData) {
        String s1 = page.getJson().jsonPath($_type + ".message").get();
        if (!s1.equals("success")){
            return;
        }
        KVTag map = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("message");
        crawlerRecord.tagsCreator().bizTags().addCustomKV("message",null);
        Map<String,String> val = (Map<String, String>) map.getVal();
        String brandName = val.get("brandName");
        String brandId = val.get("brandId");
        String brandUrl = val.get("brandUrl");
        String carTypeName = val.get("carTypeName");
        String carTypeUrl = val.get("carTypeUrl");
        String carTypeId = val.get("carTypeId");
        String koubeiUrl = val.get("koubeiUrl");
        String wendaUrl = val.get("wendaUrl");
        String s = page.getJson().jsonPath($_type + ".data").get();
        JSONObject jsonObject = JSONObject.parseObject(s);
        JSONArray onSaleCarList = jsonObject.getJSONArray("onSaleCarList");//在售款
        JSONArray stopSaleCarList = jsonObject.getJSONArray("stopSaleCarList");//停售款
        if(onSaleCarList.size()>0){
            for (Object obj : onSaleCarList){
                JSONObject object = JSONObject.parseObject((String) obj);
                JSONArray powerList = object.getJSONArray("powerList");
                for (Object ect : powerList){
                    JSONObject car = JSONObject.parseObject((String) ect);
                    JSONArray carList = car.getJSONArray("carList");
                    for (Object o : carList){
                        JSONObject jsonObject1 = JSONObject.parseObject((String) o);
                        String id = jsonObject1.getString("id");
                        String name = jsonObject1.getString("name");
                        String year = jsonObject1.getString("year");
                        year = year + "款";
                        name = name + year;
                        CrawlerData crawlerData = CrawlerData.builder()
                                .data(crawlerRecord,page)
                                .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,id))
                                .releaseTime(System.currentTimeMillis())
                                .url(crawlerRecord.getHttpRequest().getUrl())
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .addContentKV(AutoVMCommonField.Field_Brand_url,brandUrl)
                                .addContentKV(AutoVMCommonField.Field_Brand_name,brandName)
                                .addContentKV(AutoVMCommonField.Field_Brand_id,brandId)
                                .addContentKV(AutoVMCommonField.Field_Series_name,carTypeName)
                                .addContentKV(AutoVMCommonField.Field_Series_url,carTypeUrl)
                                .addContentKV(AutoVMCommonField.Field_Series_id,carTypeId)
                                .addContentKV(AutoVMCommonField.Tag_Is_sale,"1")
                                .addContentKV(AutoVMCommonField.Field_Praise_url,koubeiUrl)
                                .addContentKV(AutoVMCommonField.Field_Forum_url,wendaUrl)
                                .addContentKV(AutoVMCommonField.Field_Forum_name,carTypeName+"论坛")
                                .addContentKV(AICCommonField.Field_Content,name)
                                .flowInPipelineTag("kafka")
                                .build();
                        crawlerData.setFilterPipelineResult(true);
                        listData.add(crawlerData);
                    }
                }
            }
        }

        if(stopSaleCarList.size()>0){
            for (Object obj : onSaleCarList){
                JSONObject object = JSONObject.parseObject((String) obj);
                JSONArray powerList = object.getJSONArray("powerList");
                for (Object ect : powerList){
                    JSONObject car = JSONObject.parseObject((String) ect);
                    JSONArray carList = car.getJSONArray("carList");
                    for (Object o : carList){
                        JSONObject jsonObject1 = JSONObject.parseObject((String) o);
                        String id = jsonObject1.getString("id");
                        String name = jsonObject1.getString("name");
                        String year = jsonObject1.getString("year");
                        year = year + "款";
                        name = name + year;
                        CrawlerData crawlerData = CrawlerData.builder()
                                .data(crawlerRecord,page)
                                .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,id))
                                .releaseTime(System.currentTimeMillis())
                                .url(crawlerRecord.getHttpRequest().getUrl())
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .addContentKV(AutoVMCommonField.Field_Brand_url,brandUrl)
                                .addContentKV(AutoVMCommonField.Field_Brand_name,brandName)
                                .addContentKV(AutoVMCommonField.Field_Brand_id,brandId)
                                .addContentKV(AutoVMCommonField.Field_Series_name,carTypeName)
                                .addContentKV(AutoVMCommonField.Field_Series_url,carTypeUrl)
                                .addContentKV(AutoVMCommonField.Field_Series_id,carTypeId)
                                .addContentKV(AutoVMCommonField.Tag_Is_sale,"0")
                                .addContentKV(AutoVMCommonField.Field_Praise_url,koubeiUrl)
                                .addContentKV(AutoVMCommonField.Field_Forum_url,wendaUrl)
                                .addContentKV(AutoVMCommonField.Field_Forum_name,carTypeName+"论坛")
                                .addContentKV(AICCommonField.Field_Content,name)
                                .flowInPipelineTag("kafka")
                                .build();
                        crawlerData.setFilterPipelineResult(true);
                        listData.add(crawlerData);
                    }
                }
            }
        }
    }


    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return crawlerRecord.tagsCreator().bizTags().site().equals(site);
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public String domain() {
        return domain;
    }

    private static void recordAgainRequest(CrawlerRequestRecord crawlerRequestRecord,List<CrawlerRequestRecord> parseList){
        int count = 0;
        String url = crawlerRequestRecord.getHttpRequest().getUrl();
        if (crawlerRequestRecord.tagsCreator().bizTags().hasKVTag(RECORD_AGAIN_REQUEST)){
            count = Integer.valueOf(crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(RECORD_AGAIN_REQUEST));
            if (count >= 10){
                log.error("url excessive number of repeated downloads this url = "+url);
            }
        }else{
            count = 1;
        }
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .releaseTime(System.currentTimeMillis())
                .httpUrl(url)
                .recordKey(crawlerRequestRecord.getRecordKey()+count)
                .copyResultTags()
                .copyBizTags()
                .build();

        record.getHttpRequest().setCookies(crawlerRequestRecord.getHttpRequest().getCookies());
        record.getHttpRequest().setExtras(crawlerRequestRecord.getHttpRequest().getExtras());
        record.getHttpRequest().setHeaders(crawlerRequestRecord.getHttpRequest().getHeaders());
        record.setNeedParsedPage(crawlerRequestRecord.isNeedParsedPage());
        record.setNeedWashPage(crawlerRequestRecord.isNeedWashPage());
        record.tagsCreator().bizTags().addCustomKV(RECORD_AGAIN_REQUEST,count++);
        parseList.add(record);

    }
    /**
     * 判断是否在时间范围内
     * @param crawlerRequestRecord
     * @return
     */
    private boolean isDateRange(CrawlerRequestRecord crawlerRequestRecord,Long releaseTimeToLong){
        boolean isRange = false;
        CrawlerEnum.CrawlerRecordFilter filter = crawlerRequestRecord.getFilter();
        if (filter == CrawlerEnum.CrawlerRecordFilter.keyOrDateRange || filter == CrawlerEnum.CrawlerRecordFilter.dateRange) {
            List<FilterInfo> filterInfos = crawlerRequestRecord.getFilterInfos();
            Long startTime = null;
            Long endTime = null;
            for (FilterInfo filterInfo : filterInfos) {
                if (filterInfo.getFilter() == CrawlerEnum.CrawlerRecordFilter.dateRange) {
                    long[] dateAllowRange = filterInfo.getDateAllowRange();
                    int hourFromNow = filterInfo.getHourFromNow();
                    if (dateAllowRange != null) {
                        startTime = dateAllowRange[0];
                        endTime = dateAllowRange[1];
                    }else if(hourFromNow != 0){
                        endTime = System.currentTimeMillis()-60000;//系统时间减去一分钟
                        startTime = endTime - DateUtils.MILLIS_PER_HOUR * hourFromNow;
                    }
                }
            }
            if(startTime != null && releaseTimeToLong != 0 && startTime <= releaseTimeToLong && releaseTimeToLong <= endTime){
                isRange = true;
            }
        }else{
            isRange = true;
        }
        return isRange;
    }
}
